package v2;

import java.util.Date;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.log4j.Logger;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;
import vo.SkuInfoVO;

public class TmPageProcessor implements PageProcessor {
	
	private static final Logger log = Logger.getLogger(TmPageProcessor.class);

	private Site site = Site.me().setCharset("gbk")
			.setUserAgent("Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36")
			.addHeader("path", "/search_product.htm?sort=s&style=g&from=sn_1_rightnav&active=1&cat=50024400&brand=28247&search_condition=23")
			.addHeader("accept-encoding", "gzip,deflate,sdch,br")
			.addHeader("accept-language", "zh-CN,zh;q=0.8")
			.addHeader("upgrade-insecure-requests", "1")
			.addHeader("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
			.addHeader("authority", "list.tmall.com")
			.addHeader("scheme", "https")
			.addHeader("method", "GET")
			.addHeader("referer", "https://detail.tmall.com")
			.addCookie("cna", "A8XldfS0TMw8yCjERLg4gBCEPWQ8%3D%7CVmhIGCUFOBgkGiMXNwk1ADsbJxkiGTkDOA0tES8ULw81Cj95DvrrN2sCAaskrdMVu1bg")
			.addCookie("t", "12139ac7fafa84343e4b8cbefda6b013")
			.addCookie("_tb_token_", "xKuxQ8h6vZ3B")
			.addCookie("cookie2", "f068790ed13e0a1851d77181e1ba2d78")
			.addCookie("cq", "ccp%3D1")
			.addCookie("l", "AvT0KzveGSchk1RUeLXnjRJwRLhmwRiz")
			.addCookie("pnm_cku822", "136UW5TcyMNYQwiAiwQRHhBfEF8QXtHcklnMWc%3D%7CUm5OcktwS3ZDf0Z5QndJdSM%3D%7CU2xMHDJ7G2AHYg8hAS8XKwUlC1c2UDxbJV9xJ3E%3D%7CVGhXd1llXGdcYVRoUW5VYF5iVWhKfkJ9SHRMdU90QXpOck5ySmQy%7CVWldfS0TMw8yCjERLg4gBCEPWQ8%3D%7CVmhIGCUFOBgkGiMXNwk1ADsbJxkiGTkDOA0tES8ULw81Cj9pPw%3D%3D%7CV25Tbk5zU2xMcEl1VWtTaUlwJg%3D%3D")
			.addCookie("res", "scroll%3A1349*3186-client%3A1349*622-offset%3A1349*3186-screen%3A1366*768");
			
	private int pageTotal;

	private int pageSize = 60;// 天猫的每页60个商品

	private String entryUrl;
	
	private int num;

	public String getEntryUrl() {
		return entryUrl;
	}

	public void setEntryUrl(String entryUrl) {
		this.entryUrl = entryUrl;
	}

	public int getNum() {
		return num;
	}

	public void setNum(int num) {
		this.num = num;
	}

	@Override
	public void process(Page page){
		String url = page.getUrl().toString();
		if (url.startsWith("https://list.tmall.com/search_product.htm")) {
			if(pageTotal == 0){
				String pageTotalStr = page.getHtml().xpath("//form[@name='filterPageForm']/allText()").toString();
				// 确定总页数
				String regEx = "[^0-9]";
				Pattern p = Pattern.compile(regEx);
				Matcher m = p.matcher(pageTotalStr);
				pageTotal = Integer.valueOf(m.replaceAll("").trim());
				// 根据页数，添加需要的url
				for (int i = 1; i < pageTotal; i++) {
					Request request = new Request();
					request.setUrl(entryUrl+"&s="+i*pageSize);
					page.addTargetRequest(request);
				}
			}
			// 搜索获取商品列表
			searchGdsList(page);
		} else if (url.contains("item.htm")) {
			// 获取sku的相关信息
			genSkuInfo(page);
		}else if (url.contains("ext-mdskip.taobao.com")){
			//添加其它商家的手机地址
			addOtherShopsInfo(page);
		}
	}
	
	/**
	 * 搜索获取商品列表
	 * @param page
	 */
	private void searchGdsList(Page page) {
		Selectable xpath = page.getHtml().xpath("//*[@class='product-iWrap']");
		List<Selectable> nodes = xpath.nodes();
		for (Selectable s : nodes) {
			String urla = s.xpath("//*[@class='productImg']/@href").get();
			if(urla == null){
				continue;
			}
			Request request = new Request(urla).setPriority(0);
			page.addTargetRequest(request);
		}
		page.setSkip(true);
	}

	/**
	 * 获取sku的相关信息
	 * @param page
	 */
	private void genSkuInfo(Page page){
		List<String> scriptSpiderList = page.getHtml().xpath("//script").regex(".*noSkipMode.*").all();
		String str = scriptSpiderList.get(0);
		String[] strSplit = str.split("\n");
		
		List<Selectable> nodes = page.getHtml().xpath("//*[@id='J_Attrs']/table[1]/tbody/tr").nodes();
		StringBuilder sb = new StringBuilder();
		for(Selectable s : nodes){
			String key = s.xpath("//th/allText()").get();
			String value = s.xpath("//td/allText()").get();
			if (key == null || value == null){
				continue;
			}
			sb.append(key.trim() + ":" + value.trim().replace(";", "_").replace(":", "_") + ";");
		}
		page.putField("detail", sb.toString());
		
		for (String line : strSplit) {
			if (line.contains("\"api\"")) {
				try {
					JSONObject jsonObject = JSONObject.parseObject(line.trim());
					JSONObject jsonItemDO = JSONObject.parseObject(jsonObject.getString("itemDO"));
					String title = jsonItemDO.getString("title");
					
					JSONObject jsonValItemInfo = JSONObject.parseObject(jsonObject.getString("valItemInfo"));

					JSONObject jsonSkuMap = jsonValItemInfo.getJSONObject("skuMap");
					JSONArray jsonSkuList = jsonValItemInfo.getJSONArray("skuList");
					// 抽取json串中skuMap与skuList中的数据
					JSONArray jsonResult = getSkuInfoListJson(jsonSkuList, jsonSkuMap, title, page);
					// 输出
					page.putField("skuInfos", jsonResult);
					
					//获取其它商家的手机信息
					String spuId = jsonItemDO.getString("spuId");
					if(spuId != null){
						page.addTargetRequest(new Request("https://ext-mdskip.taobao.com/extension/initExtra.htm?spuId="+spuId));
					}
				} catch (Exception e) {
					log.error("json解析失败");
				}
				break;// 找到该行数据时，就停止循环
			}
		}
	}

	/**
	 * 抽取json串中skuMap与skuList中的数据
	 * @param jsonSkuList
	 * @param jsonSkuMap
	 * @param title
	 * @param page
	 * @return
	 */
	private JSONArray getSkuInfoListJson(JSONArray jsonSkuList, JSONObject jsonSkuMap, String title, Page page) {
		JSONArray jsonResult = new JSONArray();
		for (int i = 0; i < jsonSkuList.size(); i++) {
			SkuInfoVO skuInfoVO = new SkuInfoVO();
			JSONObject sku1 = JSONObject.parseObject(jsonSkuList.get(i).toString());
			String skuId = sku1.getString("skuId");
			String names = sku1.getString("names");
			String pvs = sku1.getString("pvs");
			skuInfoVO.setTitle(title);
			skuInfoVO.setSkuId(skuId);
			skuInfoVO.setNames(names);
			skuInfoVO.setPvs(pvs);

			JSONObject jsonSkuInfo2 = jsonSkuMap.getJSONObject(";" + pvs + ";");
			String price = jsonSkuInfo2.get("price").toString();
			skuInfoVO.setPrice(price);

			String url = page.getRequest().getUrl().toString();
			String id = url.substring(url.indexOf("?id=") + 4, url.indexOf("&", 1));
			skuInfoVO.setId(id);
			jsonResult.add(skuInfoVO);
		}
		num++;
		return jsonResult;
	}
	
	/**
	 * 添加其它商家的手机地址
	 * @param page
	 */
	private void addOtherShopsInfo(Page page){
		JSONObject jsonObject = JSONObject.parseObject(page.getJson().get());
		JSONObject jsonObject2 = JSONObject.parseObject(jsonObject.getString("similarShop"));
		JSONArray jsonArray = JSONArray.parseArray(jsonObject2.getString("shops"));
		if(jsonArray != null && jsonArray.size() > 0){
			for(int i=0;i<jsonArray.size();i++){
				JSONObject shopObj = jsonArray.getJSONObject(i);
				String skuUrl = shopObj.getString("commentUrl");
				if(skuUrl != null){
					page.addTargetRequest(new Request("https:"+skuUrl));
				}
			}
		}
		page.setSkip(true);
	}

	@Override
	public Site getSite() {
		return site;
	}

	public static void main(String[] args) throws Exception {
		String url = "https://list.tmall.com/search_product.htm?sort=s&style=g&from=sn_1_rightnav&active=1&cat=50024400&brand=28247&search_condition=23#J_crumbs";
		long startTime = new Date().getTime();
		TmPageProcessor tianMaoPageProcessor = new TmPageProcessor();
		tianMaoPageProcessor.setEntryUrl(url);
		
		String filePath = "D://baojia//v2//tm//";
		String fileName = "OPPO3.json";
		Spider spider = Spider.create(tianMaoPageProcessor);
		spider.addUrl(url);
		spider.run();
		long endTime = new Date().getTime();
		System.out.println(tianMaoPageProcessor.getNum());
		System.out.println("时间：" + String.valueOf((endTime - startTime)));

	}
}
